﻿/*
 *  File: CDataCrawler.cs
 *  --------------------- 
 *  This class provides means for accessing html source from a certain url and extract necessary 
 *  information from the tags. 
 * 
 */

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using HtmlAgilityPack;

namespace DataCrawler
{
    public class CDataCrawler
    {
        // ----- Protected section -----
        
        // DOM tree;
        protected HtmlDocument document;

        // Handler. This method will be called in the constructor in order to handle html after
        // it was get from the source. Derivered class should override this method.
        protected virtual void Handle() 
        {
        }

        // ----- Public section -----

        // Get html source from url and store in htmlData.
        public CDataCrawler(string url)
        {
            // This boolean value will return true if there's no exception during the getting process.
            bool isWorkDone = false;

            HtmlWeb web = new HtmlWeb();
            // This loop will end if there's no exception occured.
            do
            {
                try
                {
                    document = web.Load(url);
                    isWorkDone = true;
                }
                catch (Exception e)
                {
                    Console.WriteLine(e.ToString());
                }
            } while (isWorkDone != true);
            
            // Call Handle() in order to process html parsing.
            Handle();
        }
    }
}
